{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2Tokenizer, AutoModel, GPT2TokenizerFast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt2_tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "gpt2_tokenizer_fast = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n",
    "gpt2_model = AutoModel.from_pretrained(\"gpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I love my dog, because he is brave.'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"I love my dog, because he is brave.\"\n",
    "# text = \"[SOS] \" + text + \" [EOS]\"\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding = gpt2_tokenizer.encode(text, add_special_tokens=False)\n",
    "encoding_fast = gpt2_tokenizer_fast.encode(text, add_special_tokens=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('I love my dog, because he is brave.', 'I love my dog, because he is brave.')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoding = gpt2_tokenizer.decode(encoding)\n",
    "decoding_fast = gpt2_tokenizer_fast.decode(encoding_fast)\n",
    "decoding, decoding_fast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input IDs: tensor([[ 1212,   318,   281,  1672,  6827,    13],\n",
      "        [ 6610,  1672,  6827,    13, 50256, 50256],\n",
      "        [   32,  1790,   530,    13, 50256, 50256]])\n",
      "Attention masks: tensor([[1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 0, 0]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['This is an example sentence.', 'Another example sentence.', 'A short one.']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer\n",
    "\n",
    "# Instantiate the tokenizer\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "\n",
    "# Set the padding token to be the same as the end-of-sequence (EOS) token\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "# Create a list of sentences\n",
    "sentences = [\n",
    "    \"This is an example sentence.\",\n",
    "    \"Another example sentence.\",\n",
    "    \"A short one.\",\n",
    "]\n",
    "\n",
    "# Use the tokenizer to encode and pad the sentences\n",
    "encoded_batch = tokenizer.batch_encode_plus(\n",
    "    sentences,\n",
    "    padding=True,  # Enables padding\n",
    "    return_tensors=\"pt\",  # Returns PyTorch tensors (use \"tf\" for TensorFlow tensors)\n",
    ")\n",
    "\n",
    "# Access the padded input IDs and attention masks\n",
    "input_ids = encoded_batch[\"input_ids\"]\n",
    "attention_masks = encoded_batch[\"attention_mask\"]\n",
    "\n",
    "print(\"Input IDs:\", input_ids)\n",
    "print(\"Attention masks:\", attention_masks)\n",
    "\n",
    "# batch decode where attention_mask is used to ignore padding tokens\n",
    "tokenizer.batch_decode(input_ids, skip_special_tokens=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bert tokenizer and fast\n",
    "from transformers import BertTokenizer, BertTokenizerFast\n",
    "\n",
    "bert_tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\", do_lower_case=True)\n",
    "bert_tokenizer_fast = BertTokenizerFast.from_pretrained(\n",
    "    \"bert-base-uncased\", do_lower_case=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I love my dog, because he is brave.'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"I love my dog, because he is brave.\"\n",
    "# text = \"[SOS] \" + text + \" [EOS]\"\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "encodings = bert_tokenizer.encode(text, add_special_tokens=True)\n",
    "encodings_fast = bert_tokenizer_fast.encode(text, add_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 1045, 2293, 2026, 3899, 1010, 2138, 2002, 2003, 9191, 1012, 102]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[SEP]'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_tokenizer.decode([102])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input IDs: tensor([[ 101, 2023, 2003, 2019, 2742, 6251, 1012,  102],\n",
      "        [ 101, 2178, 2742, 6251, 1012,  102,    0,    0],\n",
      "        [ 101, 1037, 2460, 2028, 1012,  102,    0,    0]])\n",
      "Attention masks: tensor([[1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 0, 0]])\n",
      "['this is an example sentence.', 'another example sentence.', 'a short one.']\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "# Instantiate the tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "# Create a list of sentences\n",
    "sentences = [\n",
    "    \"This is an example sentence.\",\n",
    "    \"Another example sentence.\",\n",
    "    \"A short one.\",\n",
    "]\n",
    "\n",
    "# Use the tokenizer to encode and pad the sentences\n",
    "encoded_batch = tokenizer.batch_encode_plus(\n",
    "    sentences,\n",
    "    padding=True,  # Enables padding\n",
    "    return_tensors=\"pt\",  # Returns PyTorch tensors (use \"tf\" for TensorFlow tensors)\n",
    ")\n",
    "\n",
    "# Access the padded input IDs and attention masks\n",
    "input_ids = encoded_batch[\"input_ids\"]\n",
    "attention_masks = encoded_batch[\"attention_mask\"]\n",
    "\n",
    "print(\"Input IDs:\", input_ids)\n",
    "print(\"Attention masks:\", attention_masks)\n",
    "\n",
    "# decoding\n",
    "decoding = tokenizer.batch_decode(input_ids, skip_special_tokens=True)\n",
    "print(decoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cf67e5a265b4568a3694be758562d80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"gpt2\", use_fast=True)\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "sentences = [\n",
    "    \"This is an example sentence.\",\n",
    "    \"Another example sentence.\",\n",
    "    \"A short one.\",\n",
    "    \"This is an example sentence.\",\n",
    "    \"Another example sentence.\",\n",
    "    \"A short one.\",\n",
    "]\n",
    "\n",
    "labels = [[1, 1, 1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1], [1, 1, 1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'src'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39msrc\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mdata\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcomponents\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mdatasets\u001b[39;00m \u001b[39mimport\u001b[39;00m encode_and_pad_batch, TokenTaggingDataset\n\u001b[1;32m      3\u001b[0m dataset \u001b[39m=\u001b[39m TokenTaggingDataset(sentences, labels, tokenizer)\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'src'"
     ]
    }
   ],
   "source": [
    "from src.data.components.datasets import encode_and_pad_batch, TokenTaggingDataset\n",
    "\n",
    "dataset = TokenTaggingDataset(sentences, labels, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "dataloader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=2,\n",
    "    shuffle=True,\n",
    "    collate_fn=lambda batch: encode_and_pad_batch(batch, tokenizer),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text \n",
      " This is an example sentence.\n",
      "word encodings \n",
      " [[2023], [2003], [2019], [2742], [6251, 1012]]\n",
      "word_to_token\n",
      " [[0], [1], [2], [3], [4, 5]]\n",
      "#tokens \n",
      " 6\n",
      "algined tokens \n",
      " [2023, 2003, 2019, 2742, 6251, 1012]\n",
      "aligned decoded\n",
      " this is an example sentence.\n",
      "text \n",
      " Another example sentence.\n",
      "word encodings \n",
      " [[2178], [2742], [6251, 1012]]\n",
      "word_to_token\n",
      " [[0], [1], [2, 3]]\n",
      "#tokens \n",
      " 4\n",
      "algined tokens \n",
      " [2178, 2742, 6251, 1012]\n",
      "aligned decoded\n",
      " another example sentence.\n",
      "Input IDs:\n",
      " tensor([[ 101, 2023, 2003, 2019, 2742, 6251, 1012,  102],\n",
      "        [ 101, 2178, 2742, 6251, 1012,  102,    0,    0]])\n",
      "Decoded input:\n",
      " ['[CLS] this is an example sentence. [SEP]', '[CLS] another example sentence. [SEP] [PAD] [PAD]']\n",
      "Attention masks:\n",
      " tensor([[1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 0, 0]])\n",
      "Padded labels:\n",
      " tensor([[   1,    1,    1,    1,    1,    1, -999, -999],\n",
      "        [   1,    1,    1,    1, -999, -999, -999, -999]])\n",
      "Outputs:\n",
      " tensor([[[-0.3774, -0.3350, -0.3206,  ..., -0.5255,  0.2590,  0.6877],\n",
      "         [-0.8629, -0.6322, -0.4241,  ..., -0.5824,  0.7432,  0.1259],\n",
      "         [-0.2213, -0.9393,  0.3523,  ..., -0.3219,  0.4667,  0.5915],\n",
      "         ...,\n",
      "         [-0.0471, -0.1261,  0.0237,  ..., -0.2642,  0.1113, -0.0666],\n",
      "         [ 0.7917,  0.0632, -0.6609,  ...,  0.3360, -0.7243, -0.2557],\n",
      "         [-0.1416, -0.3367, -0.3314,  ..., -0.2132, -0.4376,  0.4454]],\n",
      "\n",
      "        [[-0.2924, -0.3253, -0.3886,  ..., -0.2868,  0.2157,  0.6433],\n",
      "         [-0.0338, -1.0024, -0.3005,  ..., -0.1190,  0.9047,  0.2311],\n",
      "         [-0.4415, -0.2425, -0.7587,  ..., -0.6208,  0.1528,  0.0825],\n",
      "         ...,\n",
      "         [ 1.0391, -0.0324, -0.4098,  ...,  0.2882, -0.8480, -0.1672],\n",
      "         [-0.3579, -0.4596, -0.1465,  ..., -0.0685,  0.0276,  0.2554],\n",
      "         [-0.3476, -0.5998, -0.2705,  ...,  0.0828,  0.1289,  0.2535]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "Outputs shape:\n",
      " torch.Size([2, 8, 768])\n",
      "text \n",
      " A short one.\n",
      "word encodings \n",
      " [[1037], [2460], [2028, 1012]]\n",
      "word_to_token\n",
      " [[0], [1], [2, 3]]\n",
      "#tokens \n",
      " 4\n",
      "algined tokens \n",
      " [1037, 2460, 2028, 1012]\n",
      "aligned decoded\n",
      " a short one.\n",
      "text \n",
      " What a long sentence this here is incredible.\n",
      "word encodings \n",
      " [[2054], [1037], [2146], [6251], [2023], [2182], [2003], [9788, 1012]]\n",
      "word_to_token\n",
      " [[0], [1], [2], [3], [4], [5], [6], [7, 8]]\n",
      "#tokens \n",
      " 9\n",
      "algined tokens \n",
      " [2054, 1037, 2146, 6251, 2023, 2182, 2003, 9788, 1012]\n",
      "aligned decoded\n",
      " what a long sentence this here is incredible.\n",
      "Input IDs:\n",
      " tensor([[ 101, 1037, 2460, 2028, 1012,  102,    0,    0,    0,    0,    0],\n",
      "        [ 101, 2054, 1037, 2146, 6251, 2023, 2182, 2003, 9788, 1012,  102]])\n",
      "Decoded input:\n",
      " ['[CLS] a short one. [SEP] [PAD] [PAD] [PAD] [PAD] [PAD]', '[CLS] what a long sentence this here is incredible. [SEP]']\n",
      "Attention masks:\n",
      " tensor([[1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "Padded labels:\n",
      " tensor([[   1,    1,    1,    1, -999, -999, -999, -999, -999, -999, -999],\n",
      "        [   1,    1,    1,    1,    1,    1,    1,    1,    1, -999, -999]])\n",
      "Outputs:\n",
      " tensor([[[-0.1449, -0.5376, -0.7223,  ..., -0.1883,  0.2320,  0.4892],\n",
      "         [ 0.4290, -0.1785, -0.9902,  ..., -0.9439,  0.1389,  0.7930],\n",
      "         [ 0.2519,  0.0264, -0.3542,  ..., -0.6387,  0.0040, -0.1435],\n",
      "         ...,\n",
      "         [ 0.1282, -0.2647, -0.0093,  ..., -0.0088,  0.0632,  0.0632],\n",
      "         [-0.2388, -0.8124, -0.1797,  ...,  0.4322,  0.2350, -0.1776],\n",
      "         [-0.1423, -0.8497, -0.1566,  ...,  0.3646,  0.2033, -0.1145]],\n",
      "\n",
      "        [[-0.0040, -0.0363, -0.0509,  ..., -0.1111,  0.2380,  0.5907],\n",
      "         [ 0.6698,  0.7413,  0.1565,  ..., -0.2265,  0.4102,  0.3011],\n",
      "         [ 0.0129,  0.4222,  0.5661,  ...,  0.6035,  0.6153,  0.5633],\n",
      "         ...,\n",
      "         [ 0.6282,  0.7849, -0.0963,  ...,  0.2277,  0.4458,  0.1739],\n",
      "         [ 0.3752,  0.0725, -0.3378,  ...,  0.1718, -0.3127, -0.5080],\n",
      "         [ 0.6674,  0.0567, -0.1778,  ...,  0.0407, -0.5813, -0.4047]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n",
      "Outputs shape:\n",
      " torch.Size([2, 11, 768])\n"
     ]
    }
   ],
   "source": [
    "from typing import List, Tuple, Union\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import GPT2Tokenizer, BertTokenizer, AutoTokenizer, AutoModel\n",
    "from src.data.components.datasets import encode_and_pad_batch, TokenTaggingDataset\n",
    "from omegaconf import DictConfig, OmegaConf\n",
    "\n",
    "\n",
    "# distribute_word_label_to_token function from a previous response\n",
    "\n",
    "# encode_and_pad_batch function from your message\n",
    "\n",
    "# Instantiate the tokenizer\n",
    "# tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "# tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "# tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "# tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\", do_lower_case=True, add_prefix_space=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\", use_fast=True)\n",
    "\n",
    "model = AutoModel.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "cfg = DictConfig({\"model\": \"bert-base-uncased\"})\n",
    "\n",
    "sentences = [\n",
    "    \"This is an example sentence.\",\n",
    "    \"Another example sentence.\",\n",
    "    \"A short one.\",\n",
    "    \"What a long sentence this here is incredible.\",\n",
    "]\n",
    "labels = [[1, 1, 1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1]]\n",
    "\n",
    "dataset = TokenTaggingDataset(sentences, labels, tokenizer, cfg)\n",
    "dataloader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=2,\n",
    "    collate_fn=lambda batch: encode_and_pad_batch(batch, tokenizer),\n",
    ")\n",
    "\n",
    "for input_ids, attention_masks, padded_labels in dataloader:\n",
    "    print(\"Input IDs:\\n\", input_ids)\n",
    "    print(\n",
    "        \"Decoded input:\\n\", tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n",
    "    )\n",
    "    print(\"Attention masks:\\n\", attention_masks)\n",
    "    print(\"Padded labels:\\n\", padded_labels)\n",
    "    outputs = model(input_ids, attention_mask=attention_masks).last_hidden_state\n",
    "    print(\"Outputs:\\n\", outputs)\n",
    "    print(\"Outputs shape:\\n\", outputs.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unified tokenizer function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.components.datasets import tokenize_text_with_labels\n",
    "from transformers import BertTokenizer, GPT2Tokenizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Examples with different models "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Hello, world! This is a test.\"\n",
    "labels = [0, 1, 2, 2, 2, 5]\n",
    "score_first_token = True\n",
    "relative_to_prev = False\n",
    "n_prev = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "tokenize_text_with_labels() missing 1 required positional argument: 'model_type'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 11\u001b[0m\n\u001b[1;32m      2\u001b[0m model_type \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mgpt2\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m      3\u001b[0m tokenizer \u001b[39m=\u001b[39m GPT2Tokenizer\u001b[39m.\u001b[39mfrom_pretrained(\u001b[39m\"\u001b[39m\u001b[39mgpt2\u001b[39m\u001b[39m\"\u001b[39m, add_prefix_space\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m      5\u001b[0m (\n\u001b[1;32m      6\u001b[0m     input_text,\n\u001b[1;32m      7\u001b[0m     tokenized_text,\n\u001b[1;32m      8\u001b[0m     tokenized_labels,\n\u001b[1;32m      9\u001b[0m     token_ids,\n\u001b[1;32m     10\u001b[0m     mask,\n\u001b[0;32m---> 11\u001b[0m ) \u001b[39m=\u001b[39m tokenize_text_with_labels(\n\u001b[1;32m     12\u001b[0m     text,\n\u001b[1;32m     13\u001b[0m     labels,\n\u001b[1;32m     14\u001b[0m     model_type,\n\u001b[1;32m     15\u001b[0m     score_first_token\u001b[39m=\u001b[39;49mscore_first_token,\n\u001b[1;32m     16\u001b[0m     relative_to_prev\u001b[39m=\u001b[39;49mrelative_to_prev,\n\u001b[1;32m     17\u001b[0m     n_prev\u001b[39m=\u001b[39;49mn_prev,\n\u001b[1;32m     18\u001b[0m )\n\u001b[1;32m     19\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mInput text:\u001b[39m\u001b[39m\"\u001b[39m, input_text)\n\u001b[1;32m     20\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTokenized text:\u001b[39m\u001b[39m\"\u001b[39m, tokenized_text)\n",
      "\u001b[0;31mTypeError\u001b[0m: tokenize_text_with_labels() missing 1 required positional argument: 'model_type'"
     ]
    }
   ],
   "source": [
    "# GPT2\n",
    "model_type = \"gpt2\"\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\", add_prefix_space=True)\n",
    "\n",
    "(\n",
    "    input_text,\n",
    "    tokenized_text,\n",
    "    tokenized_labels,\n",
    "    token_ids,\n",
    "    mask,\n",
    ") = tokenize_text_with_labels(\n",
    "    text,\n",
    "    labels,\n",
    "    model_type,\n",
    "    score_first_token=score_first_token,\n",
    "    relative_to_prev=relative_to_prev,\n",
    "    n_prev=n_prev,\n",
    ")\n",
    "print(\"Input text:\", input_text)\n",
    "print(\"Tokenized text:\", tokenized_text)\n",
    "print(\"Tokenized labels:\", tokenized_labels)\n",
    "print(\"Token IDs:\", token_ids)  # decode the token ids\n",
    "print(\"Mask:\", mask)\n",
    "tokenizer.decode(token_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "tokenize_text_with_labels() missing 1 required positional argument: 'model_type'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 12\u001b[0m\n\u001b[1;32m      2\u001b[0m model_type \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mbert-cased\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m      3\u001b[0m tokenizer \u001b[39m=\u001b[39m BertTokenizer\u001b[39m.\u001b[39mfrom_pretrained(\u001b[39m\"\u001b[39m\u001b[39mbert-base-cased\u001b[39m\u001b[39m\"\u001b[39m, add_special_tokens\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m      6\u001b[0m (\n\u001b[1;32m      7\u001b[0m     input_text,\n\u001b[1;32m      8\u001b[0m     tokenized_text,\n\u001b[1;32m      9\u001b[0m     tokenized_labels,\n\u001b[1;32m     10\u001b[0m     token_ids,\n\u001b[1;32m     11\u001b[0m     mask,\n\u001b[0;32m---> 12\u001b[0m ) \u001b[39m=\u001b[39m tokenize_text_with_labels(\n\u001b[1;32m     13\u001b[0m     text,\n\u001b[1;32m     14\u001b[0m     labels,\n\u001b[1;32m     15\u001b[0m     model_type,\n\u001b[1;32m     16\u001b[0m     score_first_token\u001b[39m=\u001b[39;49mscore_first_token,\n\u001b[1;32m     17\u001b[0m     relative_to_prev\u001b[39m=\u001b[39;49mrelative_to_prev,\n\u001b[1;32m     18\u001b[0m     n_prev\u001b[39m=\u001b[39;49mn_prev,\n\u001b[1;32m     19\u001b[0m )\n\u001b[1;32m     20\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mInput text:\u001b[39m\u001b[39m\"\u001b[39m, input_text)\n\u001b[1;32m     21\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTokenized text:\u001b[39m\u001b[39m\"\u001b[39m, tokenized_text)\n",
      "\u001b[0;31mTypeError\u001b[0m: tokenize_text_with_labels() missing 1 required positional argument: 'model_type'"
     ]
    }
   ],
   "source": [
    "# BERT\n",
    "model_type = \"bert-cased\"\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\", add_special_tokens=True)\n",
    "\n",
    "\n",
    "(\n",
    "    input_text,\n",
    "    tokenized_text,\n",
    "    tokenized_labels,\n",
    "    token_ids,\n",
    "    mask,\n",
    ") = tokenize_text_with_labels(\n",
    "    text,\n",
    "    labels,\n",
    "    model_type,\n",
    "    score_first_token=score_first_token,\n",
    "    relative_to_prev=relative_to_prev,\n",
    "    n_prev=n_prev,\n",
    ")\n",
    "print(\"Input text:\", input_text)\n",
    "print(\"Tokenized text:\", tokenized_text)\n",
    "print(\"Tokenized labels:\", tokenized_labels)\n",
    "print(\"Token IDs:\", token_ids)  # decode the token ids\n",
    "print(\"Mask:\", mask)  # decode the token ids\n",
    "tokenizer.decode(token_ids, ignore_special_tokens=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_tokenizer = BertTokenizer.from_pretrained(\n",
    "    \"bert-base-cased\", add_special_tokens=True\n",
    ")\n",
    "bert_tokenizer.sep_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.components.helsinki import HelsinkiProminenceExtractor\n",
    "from src.data.components.datasets import TokenTaggingDataset\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "extractor = HelsinkiProminenceExtractor(\n",
    "    \"/Users/lukas/Desktop/projects/MIT/prosody/prosody/repositories/helsinki-prosody/data\",\n",
    "    \"dev.txt\",\n",
    ")\n",
    "texts = extractor.get_all_texts()\n",
    "prominences = extractor.get_all_real_prominence()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = TokenTaggingDataset(\n",
    "    texts,\n",
    "    prominences,\n",
    "    bert_tokenizer,\n",
    "    \"bert-cased\",\n",
    "    score_first_token=True,\n",
    "    relative_to_prev=False,\n",
    "    n_prev=3,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.components.collators import collate_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "collator = partial(collate_fn, eos_token_id=102)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = DataLoader(dataset, batch_size=2, shuffle=False, collate_fn=collator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_text [\"A 'JOLLY' ART CRITIC\", 'There is a healthy bank holiday atmosphere about this book which is extremely pleasant']\n",
      "tokenized_text [['[CLS]', 'A', \"'\", 'J', '##OL', '##L', '##Y', \"'\", 'AR', '##T', 'CR', '##IT', '##IC', '[SEP]'], ['[CLS]', 'There', 'is', 'a', 'healthy', 'bank', 'holiday', 'atmosphere', 'about', 'this', 'book', 'which', 'is', 'extremely', 'pleasant', '[SEP]']]\n",
      "original_labels [[0.128, 2.454, 0.986, 0.233], [0.0, 0.164, 0.036, 2.144, 0.938, 0.091, 0.597, 0.162, 0.049, 0.669, 0.0, 0.038, 2.106, 0.076]]\n",
      "tokenized_labels tensor([[-1.0000,  0.1280,  2.4540, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "          0.9860, -1.0000,  0.2330, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
      "        [-1.0000,  0.0000,  0.1640,  0.0360,  2.1440,  0.9380,  0.0910,  0.5970,\n",
      "          0.1620,  0.0490,  0.6690,  0.0000,  0.0380,  2.1060,  0.0760, -1.0000]])\n",
      "input_ids tensor([[  101,   138,   112,   147, 13901,  2162,  3663,   112, 22133,  1942,\n",
      "         15531, 12150,  9741,   102,   102,   102],\n",
      "        [  101,  1247,  1110,   170,  8071,  3085,  7946,  6814,  1164,  1142,\n",
      "          1520,  1134,  1110,  4450, 10287,   102]])\n",
      "loss_mask tensor([[0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0],\n",
      "        [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "word_to_tokens [['A', [138], \"'JOLLY'\", [112, 147, 13901, 2162, 3663, 112], 'ART', [22133, 1942], 'CRITIC', [15531, 12150, 9741]], ['There', [1247], 'is', [1110], 'a', [170], 'healthy', [8071], 'bank', [3085], 'holiday', [7946], 'atmosphere', [6814], 'about', [1164], 'this', [1142], 'book', [1520], 'which', [1134], 'is', [1110], 'extremely', [4450], 'pleasant', [10287]]]\n",
      "input_text ['mr Quilter is entirely free from affectation of any kind', 'He rollicks through art with the recklessness of the tourist and describes its beauties with the enthusiasm of the auctioneer']\n",
      "tokenized_text [['[CLS]', 'm', '##r', 'Q', '##uilt', '##er', 'is', 'entirely', 'free', 'from', 'affect', '##ation', 'of', 'any', 'kind', '[SEP]'], ['[CLS]', 'He', 'roll', '##icks', 'through', 'art', 'with', 'the', 'reckless', '##ness', 'of', 'the', 'tourist', 'and', 'describes', 'its', 'be', '##aut', '##ies', 'with', 'the', 'enthusiasm', 'of', 'the', 'auction', '##eer', '[SEP]']]\n",
      "original_labels [[1.076, 0.472, 0.007, 2.294, 0.378, 0.071, 0.935, 0.03, 0.869, 0.224], [0.372, 3.3, 0.113, 1.387, 0.014, 0.0, 0.784, 0.0, 0.0, 1.864, 0.121, 0.829, 0.01, 1.439, 0.02, 0.0, 1.391, 0.206, 0.0, 0.7]]\n",
      "tokenized_labels tensor([[-1.0000,  1.0760, -1.0000,  0.4720, -1.0000, -1.0000,  0.0070,  2.2940,\n",
      "          0.3780,  0.0710,  0.9350, -1.0000,  0.0300,  0.8690,  0.2240, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000],\n",
      "        [-1.0000,  0.3720,  3.3000, -1.0000,  0.1130,  1.3870,  0.0140,  0.0000,\n",
      "          0.7840, -1.0000,  0.0000,  0.0000,  1.8640,  0.1210,  0.8290,  0.0100,\n",
      "          1.4390, -1.0000, -1.0000,  0.0200,  0.0000,  1.3910,  0.2060,  0.0000,\n",
      "          0.7000, -1.0000, -1.0000]])\n",
      "input_ids tensor([[  101,   182,  1197,   154, 20833,  1200,  1110,  3665,  1714,  1121,\n",
      "          6975,  1891,  1104,  1251,  1912,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102],\n",
      "        [  101,  1124,  5155, 18917,  1194,  1893,  1114,  1103, 23585,  1757,\n",
      "          1104,  1103,  7798,  1105,  4856,  1157,  1129, 24723,  1905,  1114,\n",
      "          1103, 12430,  1104,  1103, 11046,  8284,   102]])\n",
      "loss_mask tensor([[0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0],\n",
      "        [0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1,\n",
      "         1, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1]])\n",
      "word_to_tokens [['mr', [182, 1197], 'Quilter', [154, 20833, 1200], 'is', [1110], 'entirely', [3665], 'free', [1714], 'from', [1121], 'affectation', [6975, 1891], 'of', [1104], 'any', [1251], 'kind', [1912]], ['He', [1124], 'rollicks', [5155, 18917], 'through', [1194], 'art', [1893], 'with', [1114], 'the', [1103], 'recklessness', [23585, 1757], 'of', [1104], 'the', [1103], 'tourist', [7798], 'and', [1105], 'describes', [4856], 'its', [1157], 'beauties', [1129, 24723, 1905], 'with', [1114], 'the', [1103], 'enthusiasm', [12430], 'of', [1104], 'the', [1103], 'auctioneer', [11046, 8284]]]\n",
      "input_text ['To many no doubt he will seem to be somewhat blatant and bumptious but we prefer to regard him as being simply British', 'After listening so long to the Don Quixote of art to listen once to Sancho Panza is both salutary and refreshing']\n",
      "tokenized_text [['[CLS]', 'To', 'many', 'no', 'doubt', 'he', 'will', 'seem', 'to', 'be', 'somewhat', 'b', '##lat', '##ant', 'and', 'bump', '##ti', '##ous', 'but', 'we', 'prefer', 'to', 'regard', 'him', 'as', 'being', 'simply', 'British', '[SEP]'], ['[CLS]', 'After', 'listening', 'so', 'long', 'to', 'the', 'Don', 'Q', '##ui', '##x', '##ote', 'of', 'art', 'to', 'listen', 'once', 'to', 'San', '##cho', 'Pan', '##za', 'is', 'both', 'sa', '##lut', '##ary', 'and', 're', '##f', '##reshing', '[SEP]']]\n",
      "original_labels [[0.062, 2.803, 0.028, 0.32, 0.182, 0.002, 1.063, 0.0, 0.123, 0.353, 2.136, 0.002, 1.55, 0.601, 2.76, 0.453, 0.0, 0.311, 0.237, 0.237, 0.608, 1.273, 0.615], [0.477, 0.169, 0.513, 0.325, 0.555, 0.127, 0.876, 1.876, 0.045, 1.984, 0.472, 0.529, 1.769, 0.233, 0.31, 2.064, 0.307, 1.526, 1.486, 0.472, 1.778]]\n",
      "tokenized_labels tensor([[-1.0000e+00,  6.2000e-02,  2.8030e+00,  2.8000e-02,  3.2000e-01,\n",
      "          1.8200e-01,  2.0000e-03,  1.0630e+00,  0.0000e+00,  1.2300e-01,\n",
      "          3.5300e-01,  2.1360e+00, -1.0000e+00, -1.0000e+00,  2.0000e-03,\n",
      "          1.5500e+00, -1.0000e+00, -1.0000e+00,  6.0100e-01,  2.7600e+00,\n",
      "          4.5300e-01,  0.0000e+00,  3.1100e-01,  2.3700e-01,  2.3700e-01,\n",
      "          6.0800e-01,  1.2730e+00,  6.1500e-01, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00],\n",
      "        [-1.0000e+00,  4.7700e-01,  1.6900e-01,  5.1300e-01,  3.2500e-01,\n",
      "          5.5500e-01,  1.2700e-01,  8.7600e-01,  1.8760e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00,  4.5000e-02,  1.9840e+00,  4.7200e-01,\n",
      "          5.2900e-01,  1.7690e+00,  2.3300e-01,  3.1000e-01, -1.0000e+00,\n",
      "          2.0640e+00, -1.0000e+00,  3.0700e-01,  1.5260e+00,  1.4860e+00,\n",
      "         -1.0000e+00, -1.0000e+00,  4.7200e-01,  1.7780e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00]])\n",
      "input_ids tensor([[  101,  1706,  1242,  1185,  4095,  1119,  1209,  3166,  1106,  1129,\n",
      "          4742,   171, 16236,  2861,  1105, 20700,  3121,  2285,  1133,  1195,\n",
      "          9353,  1106,  7328,  1140,  1112,  1217,  2566,  1418,   102,   102,\n",
      "           102,   102],\n",
      "        [  101,  1258,  5578,  1177,  1263,  1106,  1103,  1790,   154,  6592,\n",
      "          1775, 11860,  1104,  1893,  1106,  5113,  1517,  1106,  1727,  8401,\n",
      "          6991,  3293,  1110,  1241, 21718, 25937,  3113,  1105,  1231,  2087,\n",
      "         27129,   102]])\n",
      "loss_mask tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 0, 0, 0, 0],\n",
      "        [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,\n",
      "         1, 0, 0, 1, 1, 0, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "word_to_tokens [['To', [1706], 'many', [1242], 'no', [1185], 'doubt', [4095], 'he', [1119], 'will', [1209], 'seem', [3166], 'to', [1106], 'be', [1129], 'somewhat', [4742], 'blatant', [171, 16236, 2861], 'and', [1105], 'bumptious', [20700, 3121, 2285], 'but', [1133], 'we', [1195], 'prefer', [9353], 'to', [1106], 'regard', [7328], 'him', [1140], 'as', [1112], 'being', [1217], 'simply', [2566], 'British', [1418]], ['After', [1258], 'listening', [5578], 'so', [1177], 'long', [1263], 'to', [1106], 'the', [1103], 'Don', [1790], 'Quixote', [154, 6592, 1775, 11860], 'of', [1104], 'art', [1893], 'to', [1106], 'listen', [5113], 'once', [1517], 'to', [1106], 'Sancho', [1727, 8401], 'Panza', [6991, 3293], 'is', [1110], 'both', [1241], 'salutary', [21718, 25937, 3113], 'and', [1105], 'refreshing', [1231, 2087, 27129]]]\n",
      "input_text ['About artists and their work mr Quilter has of course a great deal to say', \"Sculpture he regards as 'Painting's relation' so with the exception of a jaunty allusion to the 'rough modelling' of Tanagra figurines he hardly refers at all to the plastic arts but on painters he writes with much vigour and joviality\"]\n",
      "tokenized_text [['[CLS]', 'About', 'artists', 'and', 'their', 'work', 'm', '##r', 'Q', '##uilt', '##er', 'has', 'of', 'course', 'a', 'great', 'deal', 'to', 'say', '[SEP]'], ['[CLS]', 'Sculpture', 'he', 'regards', 'as', \"'\", 'Painting', \"'\", 's', 'relation', \"'\", 'so', 'with', 'the', 'exception', 'of', 'a', 'j', '##aunt', '##y', 'all', '##usion', 'to', 'the', \"'\", 'rough', 'modelling', \"'\", 'of', 'Tan', '##ag', '##ra', 'fi', '##gu', '##rine', '##s', 'he', 'hardly', 'refers', 'at', 'all', 'to', 'the', 'plastic', 'arts', 'but', 'on', 'painters', 'he', 'writes', 'with', 'much', 'v', '##igo', '##ur', 'and', 'j', '##ov', '##ial', '##ity', '[SEP]']]\n",
      "original_labels [[0.378, 2.134, 0.437, 1.437, 0.707, 2.783, 0.154, 1.391, 0.0, 0.468, 0.018, 0.587, 2.374, 0.006, 0.34], [1.207, 0.007, 0.406, 0.002, 1.343, 1.97, 2.902, 0.12, 0.0, 0.815, 0.0, 0.019, 0.974, 0.57, 0.025, 0.086, 0.711, 1.338, 0.0, 1.615, 0.814, 0.478, 3.258, 0.897, 0.0, 3.404, 0.057, 0.0, 0.482, 0.726, 0.175, 0.0, 1.719, 0.004, 0.516, 0.003, 1.59, 1.978, 0.0, 1.361]]\n",
      "tokenized_labels tensor([[-1.0000e+00,  3.7800e-01,  2.1340e+00,  4.3700e-01,  1.4370e+00,\n",
      "          7.0700e-01,  2.7830e+00, -1.0000e+00,  1.5400e-01, -1.0000e+00,\n",
      "         -1.0000e+00,  1.3910e+00,  0.0000e+00,  4.6800e-01,  1.8000e-02,\n",
      "          5.8700e-01,  2.3740e+00,  6.0000e-03,  3.4000e-01, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00],\n",
      "        [-1.0000e+00,  1.2070e+00,  7.0000e-03,  4.0600e-01,  2.0000e-03,\n",
      "          1.3430e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,  1.9700e+00,\n",
      "         -1.0000e+00,  2.9020e+00,  1.2000e-01,  0.0000e+00,  8.1500e-01,\n",
      "          0.0000e+00,  1.9000e-02,  9.7400e-01, -1.0000e+00, -1.0000e+00,\n",
      "          5.7000e-01, -1.0000e+00,  2.5000e-02,  8.6000e-02,  7.1100e-01,\n",
      "         -1.0000e+00,  1.3380e+00, -1.0000e+00,  0.0000e+00,  1.6150e+00,\n",
      "         -1.0000e+00, -1.0000e+00,  8.1400e-01, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00,  4.7800e-01,  3.2580e+00,  8.9700e-01,  0.0000e+00,\n",
      "          3.4040e+00,  5.7000e-02,  0.0000e+00,  4.8200e-01,  7.2600e-01,\n",
      "          1.7500e-01,  0.0000e+00,  1.7190e+00,  4.0000e-03,  5.1600e-01,\n",
      "          3.0000e-03,  1.5900e+00,  1.9780e+00, -1.0000e+00, -1.0000e+00,\n",
      "          0.0000e+00,  1.3610e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00]])\n",
      "input_ids tensor([[  101,  3517,  2719,  1105,  1147,  1250,   182,  1197,   154, 20833,\n",
      "          1200,  1144,  1104,  1736,   170,  1632,  2239,  1106,  1474,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102],\n",
      "        [  101, 19477,  1119, 12747,  1112,   112, 18189,   112,   188,  6796,\n",
      "           112,  1177,  1114,  1103,  5856,  1104,   170,   179, 24747,  1183,\n",
      "          1155, 17268,  1106,  1103,   112,  5902, 23741,   112,  1104, 13880,\n",
      "          8517,  1611, 20497, 13830,  8643,  1116,  1119,  6374,  4431,  1120,\n",
      "          1155,  1106,  1103,  5828,  3959,  1133,  1113, 15233,  1119,  6474,\n",
      "          1114,  1277,   191, 11466,  2149,  1105,   179,  3292,  2916,  1785,\n",
      "           102]])\n",
      "loss_mask tensor([[0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1,\n",
      "         1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])\n",
      "word_to_tokens [['About', [3517], 'artists', [2719], 'and', [1105], 'their', [1147], 'work', [1250], 'mr', [182, 1197], 'Quilter', [154, 20833, 1200], 'has', [1144], 'of', [1104], 'course', [1736], 'a', [170], 'great', [1632], 'deal', [2239], 'to', [1106], 'say', [1474]], ['Sculpture', [19477], 'he', [1119], 'regards', [12747], 'as', [1112], \"'Painting's\", [112, 18189, 112, 188], \"relation'\", [6796, 112], 'so', [1177], 'with', [1114], 'the', [1103], 'exception', [5856], 'of', [1104], 'a', [170], 'jaunty', [179, 24747, 1183], 'allusion', [1155, 17268], 'to', [1106], 'the', [1103], \"'rough\", [112, 5902], \"modelling'\", [23741, 112], 'of', [1104], 'Tanagra', [13880, 8517, 1611], 'figurines', [20497, 13830, 8643, 1116], 'he', [1119], 'hardly', [6374], 'refers', [4431], 'at', [1120], 'all', [1155], 'to', [1106], 'the', [1103], 'plastic', [5828], 'arts', [3959], 'but', [1133], 'on', [1113], 'painters', [15233], 'he', [1119], 'writes', [6474], 'with', [1114], 'much', [1277], 'vigour', [191, 11466, 2149], 'and', [1105], 'joviality', [179, 3292, 2916, 1785]]]\n",
      "input_text ['That there is a difference between colour and colours that an artist be he portrait painter or dramatist always reveals himself in his manner are ideas that can hardly be said to occur to him but mr Quilter really does his best and bravely faces every difficulty in modern art with the exception of mr Whistler', \"Painting he tells us is 'of a different quality to mathematics and finish in art is 'adding more fact'\"]\n",
      "tokenized_text [['[CLS]', 'That', 'there', 'is', 'a', 'difference', 'between', 'colour', 'and', 'colours', 'that', 'an', 'artist', 'be', 'he', 'portrait', 'painter', 'or', 'drama', '##tist', 'always', 'reveals', 'himself', 'in', 'his', 'manner', 'are', 'ideas', 'that', 'can', 'hardly', 'be', 'said', 'to', 'occur', 'to', 'him', 'but', 'm', '##r', 'Q', '##uilt', '##er', 'really', 'does', 'his', 'best', 'and', 'brave', '##ly', 'faces', 'every', 'difficulty', 'in', 'modern', 'art', 'with', 'the', 'exception', 'of', 'm', '##r', 'W', '##his', '##tler', '[SEP]'], ['[CLS]', 'Painting', 'he', 'tells', 'us', 'is', \"'\", 'of', 'a', 'different', 'quality', 'to', 'mathematics', 'and', 'finish', 'in', 'art', 'is', \"'\", 'adding', 'more', 'fact', \"'\", '[SEP]']]\n",
      "original_labels [[0.825, 0.35, 0.0, 0.0, 1.478, 0.501, 1.401, 0.071, 1.625, 0.583, 0.008, 1.941, 0.509, 0.802, 0.938, 1.407, 0.041, 1.42, 1.787, 0.888, 1.478, 0.0, 0.028, 1.185, 0.671, 2.633, 0.236, 0.02, 1.722, 0.018, 0.323, 0.07, 1.764, 0.0, 0.254, 2.13, 1.127, 0.247, 3.269, 0.438, 0.041, 1.801, 0.045, 1.785, 0.955, 1.173, 0.868, 0.0, 0.829, 1.68, 1.836, 0.0, 0.135, 0.544, 0.484, 1.611], [1.276, 0.0, 0.292, 0.421, 0.792, 0.345, 0.0, 1.525, 0.381, 0.012, 2.103, 0.288, 1.802, 0.373, 0.422, 0.806, 2.11, 0.989, 0.538]]\n",
      "tokenized_labels tensor([[-1.0000,  0.8250,  0.3500,  0.0000,  0.0000,  1.4780,  0.5010,  1.4010,\n",
      "          0.0710,  1.6250,  0.5830,  0.0080,  1.9410,  0.5090,  0.8020,  0.9380,\n",
      "          1.4070,  0.0410,  1.4200, -1.0000,  1.7870,  0.8880,  1.4780,  0.0000,\n",
      "          0.0280,  1.1850,  0.6710,  2.6330,  0.2360,  0.0200,  1.7220,  0.0180,\n",
      "          0.3230,  0.0700,  1.7640,  0.0000,  0.2540,  2.1300,  1.1270, -1.0000,\n",
      "          0.2470, -1.0000, -1.0000,  3.2690,  0.4380,  0.0410,  1.8010,  0.0450,\n",
      "          1.7850, -1.0000,  0.9550,  1.1730,  0.8680,  0.0000,  0.8290,  1.6800,\n",
      "          1.8360,  0.0000,  0.1350,  0.5440,  0.4840, -1.0000,  1.6110, -1.0000,\n",
      "         -1.0000, -1.0000],\n",
      "        [-1.0000,  1.2760,  0.0000,  0.2920,  0.4210,  0.7920,  0.3450, -1.0000,\n",
      "          0.0000,  1.5250,  0.3810,  0.0120,  2.1030,  0.2880,  1.8020,  0.3730,\n",
      "          0.4220,  0.8060,  2.1100, -1.0000,  0.9890,  0.5380, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
      "         -1.0000, -1.0000]])\n",
      "input_ids tensor([[  101,  1337,  1175,  1110,   170,  3719,  1206,  5922,  1105,  9849,\n",
      "          1115,  1126,  2360,  1129,  1119,  7494,  5125,  1137,  3362, 12948,\n",
      "          1579,  7189,  1471,  1107,  1117,  4758,  1132,  4133,  1115,  1169,\n",
      "          6374,  1129,  1163,  1106,  4467,  1106,  1140,  1133,   182,  1197,\n",
      "           154, 20833,  1200,  1541,  1674,  1117,  1436,  1105, 11313,  1193,\n",
      "          4876,  1451,  7262,  1107,  2030,  1893,  1114,  1103,  5856,  1104,\n",
      "           182,  1197,   160, 27516, 17966,   102],\n",
      "        [  101, 18189,  1119,  3301,  1366,  1110,   112,  1104,   170,  1472,\n",
      "          3068,  1106,  6686,  1105,  3146,  1107,  1893,  1110,   112,  5321,\n",
      "          1167,  1864,   112,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102]])\n",
      "loss_mask tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1,\n",
      "         1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0],\n",
      "        [0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "word_to_tokens [['That', [1337], 'there', [1175], 'is', [1110], 'a', [170], 'difference', [3719], 'between', [1206], 'colour', [5922], 'and', [1105], 'colours', [9849], 'that', [1115], 'an', [1126], 'artist', [2360], 'be', [1129], 'he', [1119], 'portrait', [7494], 'painter', [5125], 'or', [1137], 'dramatist', [3362, 12948], 'always', [1579], 'reveals', [7189], 'himself', [1471], 'in', [1107], 'his', [1117], 'manner', [4758], 'are', [1132], 'ideas', [4133], 'that', [1115], 'can', [1169], 'hardly', [6374], 'be', [1129], 'said', [1163], 'to', [1106], 'occur', [4467], 'to', [1106], 'him', [1140], 'but', [1133], 'mr', [182, 1197], 'Quilter', [154, 20833, 1200], 'really', [1541], 'does', [1674], 'his', [1117], 'best', [1436], 'and', [1105], 'bravely', [11313, 1193], 'faces', [4876], 'every', [1451], 'difficulty', [7262], 'in', [1107], 'modern', [2030], 'art', [1893], 'with', [1114], 'the', [1103], 'exception', [5856], 'of', [1104], 'mr', [182, 1197], 'Whistler', [160, 27516, 17966]], ['Painting', [18189], 'he', [1119], 'tells', [3301], 'us', [1366], 'is', [1110], \"'of\", [112, 1104], 'a', [170], 'different', [1472], 'quality', [3068], 'to', [1106], 'mathematics', [6686], 'and', [1105], 'finish', [3146], 'in', [1107], 'art', [1893], 'is', [1110], \"'adding\", [112, 5321], 'more', [1167], \"fact'\", [1864, 112]]]\n",
      "input_text [\"Portrait painting is a bad pursuit for an emotional artist as it destroys his personality and his sympathy however even for the emotional artist there is hope as a portrait can be converted into a picture 'by adding to the likeness of the sitter some dramatic interest or some picturesque adjunct'\", 'As for etchings they are of two kinds British and foreign']\n",
      "tokenized_text [['[CLS]', 'Portrait', 'painting', 'is', 'a', 'bad', 'pursuit', 'for', 'an', 'emotional', 'artist', 'as', 'it', 'destroys', 'his', 'personality', 'and', 'his', 'sympathy', 'however', 'even', 'for', 'the', 'emotional', 'artist', 'there', 'is', 'hope', 'as', 'a', 'portrait', 'can', 'be', 'converted', 'into', 'a', 'picture', \"'\", 'by', 'adding', 'to', 'the', 'like', '##ness', 'of', 'the', 'sit', '##ter', 'some', 'dramatic', 'interest', 'or', 'some', 'picturesque', 'ad', '##junct', \"'\", '[SEP]'], ['[CLS]', 'As', 'for', 'etc', '##hing', '##s', 'they', 'are', 'of', 'two', 'kinds', 'British', 'and', 'foreign', '[SEP]']]\n",
      "original_labels [[0.719, 0.426, 0.183, 0.0, 1.587, 0.374, 0.359, 0.262, 0.594, 0.618, 0.52, 0.001, 2.408, 0.301, 1.1, 0.074, 0.63, 0.737, 1.546, 0.301, 0.413, 0.096, 2.006, 0.177, 0.225, 0.863, 2.809, 0.639, 0.0, 1.702, 0.123, 0.021, 1.448, 0.097, 0.0, 1.138, 0.265, 2.183, 0.0, 0.001, 0.972, 0.0, 0.0, 1.831, 0.61, 4.187, 0.28, 0.321, 1.022, 1.392, 1.924], [0.609, 0.0, 2.051, 0.228, 0.111, 0.0, 2.246, 0.109, 1.361, 0.779, 0.24]]\n",
      "tokenized_labels tensor([[-1.0000e+00,  7.1900e-01,  4.2600e-01,  1.8300e-01,  0.0000e+00,\n",
      "          1.5870e+00,  3.7400e-01,  3.5900e-01,  2.6200e-01,  5.9400e-01,\n",
      "          6.1800e-01,  5.2000e-01,  1.0000e-03,  2.4080e+00,  3.0100e-01,\n",
      "          1.1000e+00,  7.4000e-02,  6.3000e-01,  7.3700e-01,  1.5460e+00,\n",
      "          3.0100e-01,  4.1300e-01,  9.6000e-02,  2.0060e+00,  1.7700e-01,\n",
      "          2.2500e-01,  8.6300e-01,  2.8090e+00,  6.3900e-01,  0.0000e+00,\n",
      "          1.7020e+00,  1.2300e-01,  2.1000e-02,  1.4480e+00,  9.7000e-02,\n",
      "          0.0000e+00,  1.1380e+00,  2.6500e-01, -1.0000e+00,  2.1830e+00,\n",
      "          0.0000e+00,  1.0000e-03,  9.7200e-01, -1.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  1.8310e+00, -1.0000e+00,  6.1000e-01,  4.1870e+00,\n",
      "          2.8000e-01,  3.2100e-01,  1.0220e+00,  1.3920e+00,  1.9240e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00],\n",
      "        [-1.0000e+00,  6.0900e-01,  0.0000e+00,  2.0510e+00, -1.0000e+00,\n",
      "         -1.0000e+00,  2.2800e-01,  1.1100e-01,  0.0000e+00,  2.2460e+00,\n",
      "          1.0900e-01,  1.3610e+00,  7.7900e-01,  2.4000e-01, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00]])\n",
      "input_ids tensor([[  101, 13954,  3504,  1110,   170,  2213,  9542,  1111,  1126,  6438,\n",
      "          2360,  1112,  1122, 22955,  1117,  5935,  1105,  1117, 12775,  1649,\n",
      "          1256,  1111,  1103,  6438,  2360,  1175,  1110,  2810,  1112,   170,\n",
      "          7494,  1169,  1129,  4213,  1154,   170,  3439,   112,  1118,  5321,\n",
      "          1106,  1103,  1176,  1757,  1104,  1103,  3465,  2083,  1199,  7271,\n",
      "          2199,  1137,  1199, 27667,  8050, 20327,   112,   102],\n",
      "        [  101,  1249,  1111,  3576,  8840,  1116,  1152,  1132,  1104,  1160,\n",
      "          7553,  1418,  1105,  2880,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102]])\n",
      "loss_mask tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0,\n",
      "         1, 1, 1, 1, 1, 1, 1, 0, 0, 0],\n",
      "        [0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "word_to_tokens [['Portrait', [13954], 'painting', [3504], 'is', [1110], 'a', [170], 'bad', [2213], 'pursuit', [9542], 'for', [1111], 'an', [1126], 'emotional', [6438], 'artist', [2360], 'as', [1112], 'it', [1122], 'destroys', [22955], 'his', [1117], 'personality', [5935], 'and', [1105], 'his', [1117], 'sympathy', [12775], 'however', [1649], 'even', [1256], 'for', [1111], 'the', [1103], 'emotional', [6438], 'artist', [2360], 'there', [1175], 'is', [1110], 'hope', [2810], 'as', [1112], 'a', [170], 'portrait', [7494], 'can', [1169], 'be', [1129], 'converted', [4213], 'into', [1154], 'a', [170], 'picture', [3439], \"'by\", [112, 1118], 'adding', [5321], 'to', [1106], 'the', [1103], 'likeness', [1176, 1757], 'of', [1104], 'the', [1103], 'sitter', [3465, 2083], 'some', [1199], 'dramatic', [7271], 'interest', [2199], 'or', [1137], 'some', [1199], 'picturesque', [27667], \"adjunct'\", [8050, 20327, 112]], ['As', [1249], 'for', [1111], 'etchings', [3576, 8840, 1116], 'they', [1152], 'are', [1132], 'of', [1104], 'two', [1160], 'kinds', [7553], 'British', [1418], 'and', [1105], 'foreign', [2880]]]\n",
      "input_text [\"And then the picture he draws of the ideal home where everything though ugly is hallowed by domestic memories and where beauty appeals not to the heartless eye but the family affections 'baby's there and the mother's work basket near the fire and the ornaments Fred brought home from India on the mantel board'\", 'It is really impossible not to be touched by so charming a description']\n",
      "tokenized_text [['[CLS]', 'And', 'then', 'the', 'picture', 'he', 'draws', 'of', 'the', 'ideal', 'home', 'where', 'everything', 'though', 'ugly', 'is', 'hall', '##owed', 'by', 'domestic', 'memories', 'and', 'where', 'beauty', 'appeals', 'not', 'to', 'the', 'heart', '##less', 'eye', 'but', 'the', 'family', 'affection', '##s', \"'\", 'baby', \"'\", 's', 'there', 'and', 'the', 'mother', \"'\", 's', 'work', 'basket', 'near', 'the', 'fire', 'and', 'the', 'ornaments', 'Fred', 'brought', 'home', 'from', 'India', 'on', 'the', 'man', '##tel', 'board', \"'\", '[SEP]'], ['[CLS]', 'It', 'is', 'really', 'impossible', 'not', 'to', 'be', 'touched', 'by', 'so', 'charming', 'a', 'description', '[SEP]']]\n",
      "original_labels [[0.131, 2.504, 0.127, 1.499, 0.074, 0.995, 0.0, 0.002, 0.455, 1.385, 0.023, 1.43, 0.406, 1.419, 0.05, 2.77, 0.11, 0.979, 1.914, 0.898, 0.168, 0.636, 0.286, 1.496, 0.043, 0.0, 0.509, 0.84, 0.427, 0.042, 1.139, 0.419, 0.915, 2.859, 0.317, 0.0, 0.152, 0.453, 1.674, 1.426, 0.0, 1.549, 0.34, 0.05, 1.984, 2.766, 0.396, 0.675, 0.083, 1.702, 0.244, 0.0, 1.071, 0.253], [0.323, 0.0, 2.693, 0.851, 2.558, 0.0, 0.003, 0.881, 0.148, 1.348, 1.399, 0.053, 1.061]]\n",
      "tokenized_labels tensor([[-1.0000e+00,  1.3100e-01,  2.5040e+00,  1.2700e-01,  1.4990e+00,\n",
      "          7.4000e-02,  9.9500e-01,  0.0000e+00,  2.0000e-03,  4.5500e-01,\n",
      "          1.3850e+00,  2.3000e-02,  1.4300e+00,  4.0600e-01,  1.4190e+00,\n",
      "          5.0000e-02,  2.7700e+00, -1.0000e+00,  1.1000e-01,  9.7900e-01,\n",
      "          1.9140e+00,  8.9800e-01,  1.6800e-01,  6.3600e-01,  2.8600e-01,\n",
      "          1.4960e+00,  4.3000e-02,  0.0000e+00,  5.0900e-01, -1.0000e+00,\n",
      "          8.4000e-01,  4.2700e-01,  4.2000e-02,  1.1390e+00,  4.1900e-01,\n",
      "         -1.0000e+00,  9.1500e-01, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "          2.8590e+00,  3.1700e-01,  0.0000e+00,  1.5200e-01, -1.0000e+00,\n",
      "         -1.0000e+00,  4.5300e-01,  1.6740e+00,  1.4260e+00,  0.0000e+00,\n",
      "          1.5490e+00,  3.4000e-01,  5.0000e-02,  1.9840e+00,  2.7660e+00,\n",
      "          3.9600e-01,  6.7500e-01,  8.3000e-02,  1.7020e+00,  2.4400e-01,\n",
      "          0.0000e+00,  1.0710e+00, -1.0000e+00,  2.5300e-01, -1.0000e+00,\n",
      "         -1.0000e+00],\n",
      "        [-1.0000e+00,  3.2300e-01,  0.0000e+00,  2.6930e+00,  8.5100e-01,\n",
      "          2.5580e+00,  0.0000e+00,  3.0000e-03,  8.8100e-01,  1.4800e-01,\n",
      "          1.3480e+00,  1.3990e+00,  5.3000e-02,  1.0610e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00, -1.0000e+00,\n",
      "         -1.0000e+00]])\n",
      "input_ids tensor([[  101,  1262,  1173,  1103,  3439,  1119,  9804,  1104,  1103,  7891,\n",
      "          1313,  1187,  1917,  1463, 10126,  1110,  2885, 12595,  1118,  4500,\n",
      "          5628,  1105,  1187,  5295, 15996,  1136,  1106,  1103,  1762,  2008,\n",
      "          2552,  1133,  1103,  1266, 12721,  1116,   112,  2963,   112,   188,\n",
      "          1175,  1105,  1103,  1534,   112,   188,  1250, 12916,  1485,  1103,\n",
      "          1783,  1105,  1103, 28034,  5291,  1814,  1313,  1121,  1726,  1113,\n",
      "          1103,  1299,  7854,  2313,   112,   102],\n",
      "        [  101,  1135,  1110,  1541,  4763,  1136,  1106,  1129,  4270,  1118,\n",
      "          1177, 14186,   170,  6136,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102,   102,   102,   102,   102,\n",
      "           102,   102,   102,   102,   102,   102]])\n",
      "loss_mask tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0],\n",
      "        [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "attention_mask tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
      "word_to_tokens [['And', [1262], 'then', [1173], 'the', [1103], 'picture', [3439], 'he', [1119], 'draws', [9804], 'of', [1104], 'the', [1103], 'ideal', [7891], 'home', [1313], 'where', [1187], 'everything', [1917], 'though', [1463], 'ugly', [10126], 'is', [1110], 'hallowed', [2885, 12595], 'by', [1118], 'domestic', [4500], 'memories', [5628], 'and', [1105], 'where', [1187], 'beauty', [5295], 'appeals', [15996], 'not', [1136], 'to', [1106], 'the', [1103], 'heartless', [1762, 2008], 'eye', [2552], 'but', [1133], 'the', [1103], 'family', [1266], 'affections', [12721, 1116], \"'baby's\", [112, 2963, 112, 188], 'there', [1175], 'and', [1105], 'the', [1103], \"mother's\", [1534, 112, 188], 'work', [1250], 'basket', [12916], 'near', [1485], 'the', [1103], 'fire', [1783], 'and', [1105], 'the', [1103], 'ornaments', [28034], 'Fred', [5291], 'brought', [1814], 'home', [1313], 'from', [1121], 'India', [1726], 'on', [1113], 'the', [1103], 'mantel', [1299, 7854], \"board'\", [2313, 112]], ['It', [1135], 'is', [1110], 'really', [1541], 'impossible', [4763], 'not', [1136], 'to', [1106], 'be', [1129], 'touched', [4270], 'by', [1118], 'so', [1177], 'charming', [14186], 'a', [170], 'description', [6136]]]\n"
     ]
    }
   ],
   "source": [
    "for i, batch in enumerate(dataloader):\n",
    "    for k, v in batch.items():\n",
    "        print(k, v)\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_tokenizer = BertTokenizer.from_pretrained(\n",
    "    \"bert-base-cased\", add_special_tokens=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoded_text = bert_tokenizer.decode(\n",
    "    batch[\"input_ids\"][0].tolist(), skip_special_tokens=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"A'JOLLY'ART CRITIC\""
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "prosody",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
